load libraries

In [1]:
import os
import cv2
import glob
import numpy as np
import pandas as pd

from keras.models import *
from keras.optimizers import *
from keras.layers import *
from keras.applications import *
from keras.preprocessing.image import *
Using TensorFlow backend.
In [2]:
dir = "/ext/Data/distracted_driver_detection/"

model_image_size = (320, 480)
fine_tune_layer = 116
final_layer = 135
visual_layer = 132
batch_size = 64

加载数据集

load train data

In [3]:
train_gen = ImageDataGenerator(
    featurewise_std_normalization=True,
    samplewise_std_normalization=False,
    rotation_range=10.,
    width_shift_range=0.05,
    height_shift_range=0.05,
    shear_range=0.1,
    zoom_range=0.1,
)
gen = ImageDataGenerator(
    featurewise_std_normalization=True,
    samplewise_std_normalization=False,
)

train_generator = train_gen.flow_from_directory(os.path.join(dir, 'train'),  model_image_size, shuffle=True, batch_size=batch_size, class_mode="categorical")
print("subdior to train type {}".format(train_generator.class_indices))
valid_generator = gen.flow_from_directory(os.path.join(dir, 'valid'),  model_image_size, shuffle=True, batch_size=batch_size, class_mode="categorical")
print("subdior to valid type {}".format(valid_generator.class_indices))
Found 22424 images belonging to 10 classes.
subdior to train type {'c0': 0, 'c1': 1, 'c2': 2, 'c3': 3, 'c4': 4, 'c5': 5, 'c6': 6, 'c7': 7, 'c8': 8, 'c9': 9}
Found 641 images belonging to 10 classes.
subdior to valid type {'c0': 0, 'c1': 1, 'c2': 2, 'c3': 3, 'c4': 4, 'c5': 5, 'c6': 6, 'c7': 7, 'c8': 8, 'c9': 9}
In [4]:
input_tensor = Input((*model_image_size, 3))
x = input_tensor
x = Lambda(xception.preprocess_input)(x)

base_model = Xception(input_tensor=x, weights='imagenet', include_top=False)

x = GlobalAveragePooling2D()(base_model.output)
x = Dropout(0.5)(x)
x = Dense(10, activation='softmax')(x)
model = Model(base_model.input, x)

print("total layer count {}".format(len(base_model.layers)))

for i in range(fine_tune_layer):
    model.layers[i].trainable = False
total layer count 133

训练模型

In [5]:
print("train_generator.samples = {}".format(train_generator.samples))
print("valid_generator.samples = {}".format(valid_generator.samples))
steps_train_sample = train_generator.samples // 128 + 1
steps_valid_sample = valid_generator.samples // 128 + 1
train_generator.samples = 22424
valid_generator.samples = 641
In [6]:
model.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy'])
model.fit_generator(train_generator, steps_per_epoch=steps_train_sample, epochs=4, validation_data=valid_generator, validation_steps=steps_valid_sample)

model.save("models/xception-imagenet-finetune{}-adam.h5".format(fine_tune_layer))
print("model saved!")
Epoch 1/4
/home/jidou/anaconda3/lib/python3.6/site-packages/keras/preprocessing/image.py:514: UserWarning: This ImageDataGenerator specifies `featurewise_std_normalization`, but it hasn'tbeen fit on any training data. Fit it first by calling `.fit(numpy_data)`.
  warnings.warn('This ImageDataGenerator specifies '
176/176 [==============================] - 199s - loss: 0.3029 - acc: 0.9083 - val_loss: 0.6591 - val_acc: 0.8125
Epoch 2/4
176/176 [==============================] - 192s - loss: 0.0693 - acc: 0.9808 - val_loss: 0.7409 - val_acc: 0.8411
Epoch 3/4
176/176 [==============================] - 192s - loss: 0.0381 - acc: 0.9882 - val_loss: 1.3377 - val_acc: 0.7445
Epoch 4/4
176/176 [==============================] - 193s - loss: 0.0367 - acc: 0.9900 - val_loss: 0.5299 - val_acc: 0.8802
model saved!
In [7]:
model.compile(optimizer=RMSprop(lr=1*0.00001), loss='categorical_crossentropy', metrics=['accuracy'])
model.fit_generator(train_generator, steps_per_epoch=steps_train_sample, epochs=6, validation_data=valid_generator, validation_steps=steps_valid_sample)

model.save("models/xception-imagenet-finetune{}.h5".format(fine_tune_layer))
print("model saved!")
Epoch 1/6
/home/jidou/anaconda3/lib/python3.6/site-packages/keras/preprocessing/image.py:514: UserWarning: This ImageDataGenerator specifies `featurewise_std_normalization`, but it hasn'tbeen fit on any training data. Fit it first by calling `.fit(numpy_data)`.
  warnings.warn('This ImageDataGenerator specifies '
176/176 [==============================] - 199s - loss: 0.0184 - acc: 0.9949 - val_loss: 0.4557 - val_acc: 0.8941
Epoch 2/6
176/176 [==============================] - 194s - loss: 0.0142 - acc: 0.9958 - val_loss: 0.3314 - val_acc: 0.9141
Epoch 3/6
176/176 [==============================] - 191s - loss: 0.0117 - acc: 0.9967 - val_loss: 0.4661 - val_acc: 0.9034
Epoch 4/6
176/176 [==============================] - 196s - loss: 0.0145 - acc: 0.9958 - val_loss: 0.3490 - val_acc: 0.9167
Epoch 5/6
176/176 [==============================] - 192s - loss: 0.0133 - acc: 0.9960 - val_loss: 0.3270 - val_acc: 0.9221
Epoch 6/6
176/176 [==============================] - 192s - loss: 0.0095 - acc: 0.9973 - val_loss: 0.4296 - val_acc: 0.9036
model saved!
In [8]:
from IPython.display import SVG
from keras.utils.vis_utils import model_to_dot
from keras.models import *

model = load_model("models/xception-imagenet-finetune{}.h5".format(fine_tune_layer))
print("load successed")

#SVG(model_to_dot(model).create(prog='dot', format='svg'))
load successed

CAM 可视化

http://cnnlocalization.csail.mit.edu/

$cam = (P-0.5)*w*output$

  • cam: 类激活图 X*X
  • P: 概率
  • output: 卷积层的输出 2048*1
  • w: 卷积核的权重 X*X*2048
In [9]:
z = zip([x.name for x in model.layers], range(len(model.layers)))
for k, v in z:
    print("{} - {}".format(k,v))
input_1 - 0
lambda_1 - 1
block1_conv1 - 2
block1_conv1_bn - 3
block1_conv1_act - 4
block1_conv2 - 5
block1_conv2_bn - 6
block1_conv2_act - 7
block2_sepconv1 - 8
block2_sepconv1_bn - 9
block2_sepconv2_act - 10
block2_sepconv2 - 11
block2_sepconv2_bn - 12
conv2d_1 - 13
block2_pool - 14
batch_normalization_1 - 15
add_1 - 16
block3_sepconv1_act - 17
block3_sepconv1 - 18
block3_sepconv1_bn - 19
block3_sepconv2_act - 20
block3_sepconv2 - 21
block3_sepconv2_bn - 22
conv2d_2 - 23
block3_pool - 24
batch_normalization_2 - 25
add_2 - 26
block4_sepconv1_act - 27
block4_sepconv1 - 28
block4_sepconv1_bn - 29
block4_sepconv2_act - 30
block4_sepconv2 - 31
block4_sepconv2_bn - 32
conv2d_3 - 33
block4_pool - 34
batch_normalization_3 - 35
add_3 - 36
block5_sepconv1_act - 37
block5_sepconv1 - 38
block5_sepconv1_bn - 39
block5_sepconv2_act - 40
block5_sepconv2 - 41
block5_sepconv2_bn - 42
block5_sepconv3_act - 43
block5_sepconv3 - 44
block5_sepconv3_bn - 45
add_4 - 46
block6_sepconv1_act - 47
block6_sepconv1 - 48
block6_sepconv1_bn - 49
block6_sepconv2_act - 50
block6_sepconv2 - 51
block6_sepconv2_bn - 52
block6_sepconv3_act - 53
block6_sepconv3 - 54
block6_sepconv3_bn - 55
add_5 - 56
block7_sepconv1_act - 57
block7_sepconv1 - 58
block7_sepconv1_bn - 59
block7_sepconv2_act - 60
block7_sepconv2 - 61
block7_sepconv2_bn - 62
block7_sepconv3_act - 63
block7_sepconv3 - 64
block7_sepconv3_bn - 65
add_6 - 66
block8_sepconv1_act - 67
block8_sepconv1 - 68
block8_sepconv1_bn - 69
block8_sepconv2_act - 70
block8_sepconv2 - 71
block8_sepconv2_bn - 72
block8_sepconv3_act - 73
block8_sepconv3 - 74
block8_sepconv3_bn - 75
add_7 - 76
block9_sepconv1_act - 77
block9_sepconv1 - 78
block9_sepconv1_bn - 79
block9_sepconv2_act - 80
block9_sepconv2 - 81
block9_sepconv2_bn - 82
block9_sepconv3_act - 83
block9_sepconv3 - 84
block9_sepconv3_bn - 85
add_8 - 86
block10_sepconv1_act - 87
block10_sepconv1 - 88
block10_sepconv1_bn - 89
block10_sepconv2_act - 90
block10_sepconv2 - 91
block10_sepconv2_bn - 92
block10_sepconv3_act - 93
block10_sepconv3 - 94
block10_sepconv3_bn - 95
add_9 - 96
block11_sepconv1_act - 97
block11_sepconv1 - 98
block11_sepconv1_bn - 99
block11_sepconv2_act - 100
block11_sepconv2 - 101
block11_sepconv2_bn - 102
block11_sepconv3_act - 103
block11_sepconv3 - 104
block11_sepconv3_bn - 105
add_10 - 106
block12_sepconv1_act - 107
block12_sepconv1 - 108
block12_sepconv1_bn - 109
block12_sepconv2_act - 110
block12_sepconv2 - 111
block12_sepconv2_bn - 112
block12_sepconv3_act - 113
block12_sepconv3 - 114
block12_sepconv3_bn - 115
add_11 - 116
block13_sepconv1_act - 117
block13_sepconv1 - 118
block13_sepconv1_bn - 119
block13_sepconv2_act - 120
block13_sepconv2 - 121
block13_sepconv2_bn - 122
conv2d_4 - 123
block13_pool - 124
batch_normalization_4 - 125
add_12 - 126
block14_sepconv1 - 127
block14_sepconv1_bn - 128
block14_sepconv1_act - 129
block14_sepconv2 - 130
block14_sepconv2_bn - 131
block14_sepconv2_act - 132
global_average_pooling2d_1 - 133
dropout_1 - 134
dense_1 - 135
In [10]:
import matplotlib.pyplot as plt
import random
%matplotlib inline
%config InlineBackend.figure_format = 'retina'

def show_heatmap_image(model_show, weights_show):
    test_dir = os.path.join(dir,  "test", "test" )
    image_files = glob.glob(os.path.join(test_dir,"*"))
    print(len(image_files))
    
    plt.figure(figsize=(12, 24))
    for i in range(10):
        plt.subplot(5, 2, i+1)
        img = cv2.imread(image_files[2000*i+113])
        img = cv2.resize(img,  (model_image_size[1],model_image_size[0]))
        x = img.copy()
        x.astype(np.float32)
        out, predictions = model_show.predict(np.expand_dims(x, axis=0))
        predictions = predictions[0]
        out = out[0]
        
        max_idx = np.argmax(predictions)
        prediction = predictions[max_idx]

        status = ["safe driving",  " texting - right",  "phone - right",  "texting - left",  "phone - left",  
                  "operation radio", "drinking", "reaching behind", "hair and makeup", "talking"]

        plt.title('c%d |%s| %.2f%%' % (max_idx , status[max_idx], prediction*100))
    
        cam = (prediction - 0.5) * np.matmul(out, weights_show)
        cam = cam[:,:,max_idx]
        cam -= cam.min()
        cam /= cam.max()
        cam -= 0.2
        cam /= 0.8

        cam = cv2.resize(cam, (model_image_size[1],model_image_size[0]))
        heatmap = cv2.applyColorMap(np.uint8(255*cam), cv2.COLORMAP_JET)
        heatmap[np.where(cam <= 0.2)] = 0

        out = cv2.addWeighted(img, 0.8, heatmap, 0.4, 0)

        plt.axis('off')
        plt.imshow(out[:,:,::-1])
print("done")
done
In [11]:
weights = model.layers[final_layer].get_weights()[0]
layer_output = model.layers[visual_layer].output
model2 = Model(model.input, [layer_output, model.output])
print("layer_output {0}".format(layer_output))
print("weights shape {0}".format(weights.shape))
show_heatmap_image(model2, weights)
layer_output Tensor("block14_sepconv2_act_1/Relu:0", shape=(?, 10, 15, 2048), dtype=float32)
weights shape (2048, 10)
79726
In [12]:
def gen_kaggle_csv(model,  model_image_size, csv_name):
    dir = "/ext/Data/distracted_driver_detection/"

    gen = ImageDataGenerator()
    test_generator = gen.flow_from_directory(dir + "test/",  model_image_size, shuffle=False, 
                                             batch_size=batch_size, class_mode=None)
#     s = test_generator.__dict__
#     del s['filenames']
#     print(s)
    y_pred = model.predict_generator(test_generator,  steps=test_generator.samples//batch_size+1,  verbose=1)
    print("y_pred shape {}".format(y_pred.shape))
    y_pred = y_pred.clip(min=0.005, max=0.995)
    print(y_pred[:3])

    l = list()
    for i, fname in enumerate(test_generator.filenames):
        name = fname[fname.rfind('/')+1:]
        l.append( [name, *y_pred[i]] )

    l = np.array(l)
    data = {'img': l[:,0]}
    for i in range(10):
        data["c%d"%i] = l[:,i+1]
    df = pd.DataFrame(data, columns=['img'] + ['c%d'%i for i in range(10)])
    df.head(10)
    df = df.sort_values(by='img')
    df.to_csv(csv_name, index=None, float_format='%.3f')
    print("csv saved")

print("done")
done
In [13]:
gen_kaggle_csv(model,  model_image_size, 'csv/Xception-imagenet-finetune{}-pred.csv'.format(fine_tune_layer))
Found 79726 images belonging to 1 classes.
1246/1246 [==============================] - 1207s     
y_pred shape (79726, 10)
[[ 0.005       0.005       0.005       0.005       0.005       0.995       0.005
   0.005       0.005       0.005     ]
 [ 0.005       0.005       0.005       0.005       0.98422527  0.005       0.005
   0.005       0.005       0.01325804]
 [ 0.005       0.005       0.005       0.02493379  0.97415757  0.005       0.005
   0.005       0.005       0.005     ]]
csv saved
In [ ]: